experiments <- c("01-adaptive_destruction_size", 
"02-adaptive_best_insertion", "03-adaptive_destruction_position", 
"04-adaptive_local_search", "05-adaptive_perturb", "06-adaptive_neighborhood_size")


perf_experiments <- c(experiments, "09-final_comparison")

perfs <- map(perf_experiments, ~{
  exp_folder <- here("reports", "aos", "data", .x)
  perf_folder <- file.path(exp_folder, "perf")
  load_perfs(perf_folder) %>%
    mutate(experiment = .x)
})
#> 
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#>   dist = col_character(),
#>   corr = col_character(),
#>   no_jobs = col_double(),
#>   no_machines = col_double(),
#>   problem = col_character(),
#>   type = col_character(),
#>   objective = col_character(),
#>   instance = col_character(),
#>   cost = col_double(),
#>   time = col_double(),
#>   no_evals = col_double()
#> )
#> 
#> 
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#>   dist = col_character(),
#>   corr = col_character(),
#>   no_jobs = col_double(),
#>   no_machines = col_double(),
#>   problem = col_character(),
#>   type = col_character(),
#>   objective = col_character(),
#>   instance = col_character(),
#>   cost = col_double(),
#>   time = col_double(),
#>   no_evals = col_double()
#> )
#> 
#> 
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#>   dist = col_character(),
#>   corr = col_character(),
#>   no_jobs = col_double(),
#>   no_machines = col_double(),
#>   problem = col_character(),
#>   type = col_character(),
#>   objective = col_character(),
#>   instance = col_character(),
#>   cost = col_double(),
#>   time = col_double(),
#>   no_evals = col_double()
#> )
#> 
#> 
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#>   dist = col_character(),
#>   corr = col_character(),
#>   no_jobs = col_double(),
#>   no_machines = col_double(),
#>   problem = col_character(),
#>   type = col_character(),
#>   objective = col_character(),
#>   instance = col_character(),
#>   cost = col_double(),
#>   time = col_double(),
#>   no_evals = col_double()
#> )
#> 
#> 
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#>   dist = col_character(),
#>   corr = col_character(),
#>   no_jobs = col_double(),
#>   no_machines = col_double(),
#>   problem = col_character(),
#>   type = col_character(),
#>   objective = col_character(),
#>   instance = col_character(),
#>   cost = col_double(),
#>   time = col_double(),
#>   no_evals = col_double()
#> )
#> 
#> 
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#>   dist = col_character(),
#>   corr = col_character(),
#>   no_jobs = col_double(),
#>   no_machines = col_double(),
#>   problem = col_character(),
#>   type = col_character(),
#>   objective = col_character(),
#>   instance = col_character(),
#>   cost = col_double(),
#>   time = col_double(),
#>   no_evals = col_double()
#> )
#> 
#> 
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#>   dist = col_character(),
#>   corr = col_character(),
#>   no_jobs = col_double(),
#>   no_machines = col_double(),
#>   problem = col_character(),
#>   type = col_character(),
#>   objective = col_character(),
#>   instance = col_character(),
#>   cost = col_double(),
#>   time = col_double(),
#>   no_evals = col_double()
#> )

names(perfs) <- perf_experiments

extra_perfs <- c("01-adaptive_destruction_size", 
"02-adaptive_best_insertion", "03-adaptive_destruction_position", 
"04-adaptive_local_search", "05-adaptive_perturb", "06-adaptive_neighborhood_size", "09-final_comparison")

extras <- map(extra_perfs, ~{
  exp_folder <- here("reports", "aos", "data", .x)
  perf_folder <- file.path(exp_folder, "extra")
  dt <- load_perfs(perf_folder) %>%
    mutate(experiment = .x)
  perfs[[.x]] <<- bind_rows(perfs[[.x]], dt)
  dt
})
#> 
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#>   dist = col_character(),
#>   corr = col_character(),
#>   no_jobs = col_double(),
#>   no_machines = col_double(),
#>   problem = col_character(),
#>   type = col_character(),
#>   objective = col_character(),
#>   instance = col_character(),
#>   cost = col_double(),
#>   time = col_double(),
#>   no_evals = col_double()
#> )
#> 
#> 
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#>   dist = col_character(),
#>   corr = col_character(),
#>   no_jobs = col_double(),
#>   no_machines = col_double(),
#>   problem = col_character(),
#>   type = col_character(),
#>   objective = col_character(),
#>   instance = col_character(),
#>   cost = col_double(),
#>   time = col_double(),
#>   no_evals = col_double()
#> )
#> 
#> 
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#>   dist = col_character(),
#>   corr = col_character(),
#>   no_jobs = col_double(),
#>   no_machines = col_double(),
#>   problem = col_character(),
#>   type = col_character(),
#>   objective = col_character(),
#>   instance = col_character(),
#>   cost = col_double(),
#>   time = col_double(),
#>   no_evals = col_double()
#> )
#> 
#> 
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#>   dist = col_character(),
#>   corr = col_character(),
#>   no_jobs = col_double(),
#>   no_machines = col_double(),
#>   problem = col_character(),
#>   type = col_character(),
#>   objective = col_character(),
#>   instance = col_character(),
#>   cost = col_double(),
#>   time = col_double(),
#>   no_evals = col_double()
#> )
#> 
#> 
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#>   dist = col_character(),
#>   corr = col_character(),
#>   no_jobs = col_double(),
#>   no_machines = col_double(),
#>   problem = col_character(),
#>   type = col_character(),
#>   objective = col_character(),
#>   instance = col_character(),
#>   cost = col_double(),
#>   time = col_double(),
#>   no_evals = col_double()
#> )
#> 
#> 
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#>   dist = col_character(),
#>   corr = col_character(),
#>   no_jobs = col_double(),
#>   no_machines = col_double(),
#>   problem = col_character(),
#>   type = col_character(),
#>   objective = col_character(),
#>   instance = col_character(),
#>   cost = col_double(),
#>   time = col_double(),
#>   no_evals = col_double()
#> )
#> 
#> 
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#>   dist = col_character(),
#>   corr = col_character(),
#>   no_jobs = col_double(),
#>   no_machines = col_double(),
#>   problem = col_character(),
#>   type = col_character(),
#>   objective = col_character(),
#>   instance = col_character(),
#>   cost = col_double(),
#>   time = col_double(),
#>   no_evals = col_double()
#> )

names(extras) <- extra_perfs 
perfs[["01-adaptive_destruction_size"]] %>%
  ggplot(aes(x = algo, fill = algo, y = arfp)) +
  geom_violin() + 
  labs(x = NULL) +
  ggtitle("Adaptive destruction size") + 
  theme_minimal()

perfs[["01-adaptive_destruction_size"]] %>%
  group_by(id) %>%
  mutate(rank = rank(fitness)) %>%
  select(algo, id, rank) %>%
  ggplot(aes(x = algo, fill = algo, y = rank)) +
  geom_violin() + 
  labs(x = NULL) +
  ggtitle("Adaptive destruction size") + 
  theme_minimal()

perfs[["01-adaptive_destruction_size"]] %>%
  group_by(algo) %>%
  summarize(
    q00 = quantile(arfp, 0.0),
    q25 = quantile(arfp, 0.25),
    q50 = quantile(arfp, 0.5),
    mean = mean(arfp),
    q75 = quantile(arfp, 0.75),
    q100 = quantile(arfp, 1.00)
  ) %>% knitr::kable(booktabs = TRUE)
algo q00 q25 q50 mean q75 q100
default -0.2634677 0 0.2164343 0.5948598 0.9918071 4.804028
epsilon_greedy -0.1425145 0 0.1785848 0.4913083 0.7952873 4.317461
frrmab -0.1218769 0 0.1057885 0.4535496 0.7135609 4.804028
linucb -0.1218769 0 0.1742060 0.4413102 0.7292240 3.904245
pm -0.1218769 0 0.1897755 0.4664731 0.8037411 4.656689
random-2_4 -0.2282447 0 0.2712795 0.6478915 1.0316041 5.609914
random-2_4_6 -0.0758921 0 0.1899424 0.5058764 0.8477862 3.719311
random-4_6 -0.1218769 0 0.1453131 0.4792023 0.8156606 4.031528
random-4_8 -0.3411266 0 0.1386972 0.4156840 0.6595141 3.948798
ts -0.2602926 0 0.1006535 0.4375860 0.6696480 4.804028
perfs[["01-adaptive_destruction_size"]] %>%
  group_by(algo) %>%
  summarize(
    q00 = quantile(arfp, 0.0),
    q25 = quantile(arfp, 0.25),
    q50 = quantile(arfp, 0.5),
    mean = mean(arfp),
    q75 = quantile(arfp, 0.75),
    q100 = quantile(arfp, 1.00)
  ) %>% knitr::kable(booktabs = TRUE)
algo q00 q25 q50 mean q75 q100
default -0.2634677 0 0.2164343 0.5948598 0.9918071 4.804028
epsilon_greedy -0.1425145 0 0.1785848 0.4913083 0.7952873 4.317461
frrmab -0.1218769 0 0.1057885 0.4535496 0.7135609 4.804028
linucb -0.1218769 0 0.1742060 0.4413102 0.7292240 3.904245
pm -0.1218769 0 0.1897755 0.4664731 0.8037411 4.656689
random-2_4 -0.2282447 0 0.2712795 0.6478915 1.0316041 5.609914
random-2_4_6 -0.0758921 0 0.1899424 0.5058764 0.8477862 3.719311
random-4_6 -0.1218769 0 0.1453131 0.4792023 0.8156606 4.031528
random-4_8 -0.3411266 0 0.1386972 0.4156840 0.6595141 3.948798
ts -0.2602926 0 0.1006535 0.4375860 0.6696480 4.804028
algo_perfs <- perfs[["01-adaptive_destruction_size"]] %>%
  filter(!(algo %in% c('random-2_4','random-2_4_6','random-4_6')))
frdAllPairsNemenyiTest(algo_perfs$arfp, algo_perfs$algo, paste(algo_perfs$id, "-", algo_perfs$seed))
#> 
#>  Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> data: y, groups and blocks
#>                default epsilon_greedy frrmab  linucb  pm      random-4_8
#> epsilon_greedy 3.1e-11 -              -       -       -       -         
#> frrmab         < 2e-16 0.00044        -       -       -       -         
#> linucb         7.1e-14 0.08580        0.75125 -       -       -         
#> pm             6.1e-14 0.71573        0.09970 0.89278 -       -         
#> random-4_8     < 2e-16 1.5e-07        0.71573 0.03851 0.00044 -         
#> ts             < 2e-16 2.2e-05        0.99571 0.33863 0.01531 0.97051
#> 
#> P value adjustment method: single-step
perfs[["02-adaptive_best_insertion"]] %>%
  ggplot(aes(x = algo, fill = algo, y = arfp)) +
  geom_violin() + 
  labs(x = NULL) +
  scale_y_continuous(trans = "pseudo_log") +
  ggtitle("Adaptive local search focus") + 
  theme_minimal()

perfs[["02-adaptive_best_insertion"]] %>%
  group_by(algo) %>%
  summarize(
    q00 = quantile(arfp, 0.0),
    q25 = quantile(arfp, 0.25),
    q50 = quantile(arfp, 0.5),
    mean = mean(arfp),
    q75 = quantile(arfp, 0.75),
    q100 = quantile(arfp, 1.00)
  ) %>% knitr::kable(booktabs = TRUE)
algo q00 q25 q50 mean q75 q100
default -0.2634677 0 0.2164343 0.5948598 0.9918071 4.804028
epsilon_greedy -0.1218769 0 0.1921976 0.5461791 0.9004010 5.214231
frrmab -0.0609385 0 0.1800011 0.5541156 0.8848130 4.843860
linucb -0.1218769 0 0.2087576 0.5696649 0.8966363 4.744624
pm -0.1218769 0 0.2205438 0.5834869 0.9316094 5.772001
ts -0.1218769 0 0.2028308 0.5555989 0.8754067 4.843860
algo_perfs <- perfs[["02-adaptive_best_insertion"]]
frdAllPairsNemenyiTest(algo_perfs$arfp, algo_perfs$algo, paste(algo_perfs$id, "-", algo_perfs$seed))
#> 
#>  Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> data: y, groups and blocks
#>                default epsilon_greedy frrmab  linucb  pm     
#> epsilon_greedy 0.00011 -              -       -       -      
#> frrmab         4.9e-05 0.99998        -       -       -      
#> linucb         0.02624 0.72538        0.61201 -       -      
#> pm             0.13486 0.33834        0.24534 0.99080 -      
#> ts             0.00020 1.00000        0.99965 0.79894 0.41537
#> 
#> P value adjustment method: single-step
perfs[["03-adaptive_destruction_position"]] %>%
  ggplot(aes(x = algo, fill = algo, y = arfp)) +
  geom_violin() + 
  labs(x = NULL) +
  ggtitle("Adaptive destruction position") + 
  theme_minimal()

perfs[["03-adaptive_destruction_position"]] %>%
  group_by(algo) %>%
  summarize(
    q00 = quantile(arfp, 0.0),
    q25 = quantile(arfp, 0.25),
    q50 = quantile(arfp, 0.5),
    mean = mean(arfp),
    q75 = quantile(arfp, 0.75),
    q100 = quantile(arfp, 1.00)
  ) %>% knitr::kable(booktabs = TRUE)
algo q00 q25 q50 mean q75 q100
default -0.2634677 0.0000000 0.2164343 0.5942982 0.9886761 4.804028
epsilon_greedy -0.1681322 0.0000000 0.2337973 0.5750158 0.9234700 4.732908
frrmab -0.1451215 0.0000000 0.2023799 0.5530878 0.9248768 4.825145
linucb -0.1218769 0.0053988 0.3602194 0.6723328 1.1170163 5.175201
pm -0.1218769 0.0000000 0.2350556 0.5632123 0.9339044 4.825145
ts -0.1218769 0.0000000 0.2144830 0.5480395 0.8876607 5.164588
algo_perfs <- perfs[["03-adaptive_destruction_position"]]
frdAllPairsNemenyiTest(algo_perfs$arfp, algo_perfs$algo, paste(algo_perfs$id, "-", algo_perfs$seed))
#> 
#>  Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> data: y, groups and blocks
#>                default epsilon_greedy frrmab  linucb  pm    
#> epsilon_greedy 0.1467  -              -       -       -     
#> frrmab         0.0039  0.8363         -       -       -     
#> linucb         0.0040  2.2e-08        6.2e-12 -       -     
#> pm             0.0976  1.0000         0.9077  7.4e-09 -     
#> ts             0.0093  0.9311         0.9999  3.6e-11 0.9698
#> 
#> P value adjustment method: single-step
perfs[["04-adaptive_local_search"]] %>%
  ggplot(aes(x = algo, fill = algo, y = arfp)) +
  geom_violin() + 
  labs(x = NULL) +
  ggtitle("Adaptive local search") + 
  theme_minimal()

perfs[["04-adaptive_local_search"]] %>%
  group_by(algo) %>%
  summarize(
    q00 = quantile(arfp, 0.0),
    q25 = quantile(arfp, 0.25),
    q50 = quantile(arfp, 0.5),
    mean = mean(arfp),
    q75 = quantile(arfp, 0.75),
    q100 = quantile(arfp, 1.00)
  ) %>% knitr::kable(booktabs = TRUE)
algo q00 q25 q50 mean q75 q100
default -0.2634677 0 0.2164343 0.5948598 0.9918071 4.804028
epsilon_greedy -0.1218769 0 0.2435542 0.6860770 1.0222437 6.712545
frrmab -0.1821084 0 0.2321387 0.6662321 1.0314004 6.195962
linucb -0.1218769 0 0.2580531 0.6963908 1.0537408 6.617857
pm -0.1730310 0 0.2321169 0.6858710 1.0607713 6.203405
random -0.1218769 0 0.2525543 0.6941423 1.0851414 6.203405
ts -0.1218769 0 0.2345952 0.6621848 1.0453204 5.473182
algo_perfs <- perfs[["04-adaptive_local_search"]] 
frdAllPairsNemenyiTest(algo_perfs$arfp, algo_perfs$algo, paste(algo_perfs$id, "-", algo_perfs$seed))
#> 
#>  Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> data: y, groups and blocks
#>                default epsilon_greedy frrmab linucb pm   random
#> epsilon_greedy 0.75    -              -      -      -    -     
#> frrmab         0.29    0.99           -      -      -    -     
#> linucb         0.12    0.92           1.00   -      -    -     
#> pm             0.35    1.00           1.00   1.00   -    -     
#> random         0.29    0.99           1.00   1.00   1.00 -     
#> ts             0.87    1.00           0.97   0.82   0.98 0.96
#> 
#> P value adjustment method: single-step
perfs[["05-adaptive_perturb"]] %>%
  ggplot(aes(x = algo, fill = algo, y = arfp)) +
  geom_violin() + 
  labs(x = NULL) +
  scale_y_continuous(trans = "pseudo_log") +
  ggtitle("Adaptive perturbation") + 
  theme_minimal()

perfs[["05-adaptive_perturb"]] %>%
  group_by(algo) %>%
  summarize(
    q00 = quantile(arfp, 0.0),
    q25 = quantile(arfp, 0.25),
    q50 = quantile(arfp, 0.5),
    mean = mean(arfp),
    q75 = quantile(arfp, 0.75),
    q100 = quantile(arfp, 1.00)
  ) %>% knitr::kable(booktabs = TRUE)
algo q00 q25 q50 mean q75 q100
default -0.2634677 0 0.2154749 0.5952273 0.9886761 4.804028
epsilon_greedy -0.5071793 0 0.0620930 0.3774206 0.5734283 3.808214
frrmab -0.3199051 0 0.0486414 0.3529366 0.5083178 4.237827
linucb -0.3199936 0 0.0978909 0.4340297 0.6175167 4.804028
pm -0.7400194 0 0.0474168 0.3511759 0.5011281 4.426239
random -0.7405741 0 0.0459770 0.3482407 0.4987108 4.426239
ts -0.5179541 0 0.0456852 0.3434712 0.4676016 4.940754
algo_perfs <- perfs[["05-adaptive_perturb"]]
frdAllPairsNemenyiTest(algo_perfs$arfp, algo_perfs$algo, paste(algo_perfs$id, "-", algo_perfs$seed))
#> 
#>  Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> data: y, groups and blocks
#>                default epsilon_greedy frrmab  linucb  pm      random 
#> epsilon_greedy 6.7e-14 -              -       -       -       -      
#> frrmab         < 2e-16 0.27973        -       -       -       -      
#> linucb         5.1e-14 0.53636        0.00102 -       -       -      
#> pm             < 2e-16 0.16353        0.99998 0.00034 -       -      
#> random         < 2e-16 0.12781        0.99982 0.00021 1.00000 -      
#> ts             < 2e-16 0.45106        0.99996 0.00321 0.99817 0.99509
#> 
#> P value adjustment method: single-step
perfs[["06-adaptive_neighborhood_size"]] %>%
  ggplot(aes(x = algo, fill = algo, y = arfp)) +
  geom_violin() + 
  labs(x = NULL) +
  ggtitle("Adaptive neighborhood size") + 
  theme_minimal()

perfs[["06-adaptive_neighborhood_size"]] %>%
  group_by(algo) %>%
  summarize(
    q00 = quantile(arfp, 0.0),
    q25 = quantile(arfp, 0.25),
    q50 = quantile(arfp, 0.5),
    mean = mean(arfp),
    q75 = quantile(arfp, 0.75),
    q100 = quantile(arfp, 1.00)
  ) %>% knitr::kable(booktabs = TRUE)
algo q00 q25 q50 mean q75 q100
default -0.2634677 0 0.2190707 0.5959512 0.9886761 4.804028
epsilon_greedy -0.0882223 0 0.2190367 0.5986173 0.9801334 4.276694
frrmab -0.1218769 0 0.1964958 0.5892183 1.0078721 4.344925
linucb -0.1098014 0 0.1903865 0.5886529 0.9714045 4.349643
pm -0.1218769 0 0.2308222 0.5929648 0.9750639 4.235910
random-10 -0.1218769 0 0.2471588 0.6459067 1.0161432 5.165043
random-2 -0.1218769 0 0.2188905 0.5665911 0.9406860 5.010496
random-3 -0.1218769 0 0.2201530 0.5902228 0.9615418 4.235910
random-5 -0.1218769 0 0.2222259 0.6046689 1.0474191 4.772052
ts -0.0882223 0 0.2185640 0.5977707 0.9728888 4.344925
algo_perfs <- perfs[["06-adaptive_neighborhood_size"]] %>%
  filter(!(algo %in% c('random-3', 'random-5', 'random-10')))
frdAllPairsNemenyiTest(algo_perfs$arfp, algo_perfs$algo, paste(algo_perfs$id, "-", algo_perfs$seed))
#> 
#>  Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> data: y, groups and blocks
#>                default epsilon_greedy frrmab linucb pm    random-2
#> epsilon_greedy 0.136   -              -      -      -     -       
#> frrmab         0.230   1.000          -      -      -     -       
#> linucb         0.303   1.000          1.000  -      -     -       
#> pm             0.644   0.971          0.994  0.998  -     -       
#> random-2       0.038   0.999          0.992  0.979  0.811 -       
#> ts             0.729   0.945          0.985  0.995  1.000 0.735
#> 
#> P value adjustment method: single-step
perfs %>%
  bind_rows() %>%
  group_by(experiment, algo) %>%
  summarise(
    arfp = sprintf("%.3f %.2f", mean(arfp), sd(arfp))
  ) %>%
  pivot_wider(names_from = "algo", values_from = "arfp") %>%
  knitr::kable(booktabs = TRUE)
#> `summarise()` has grouped output by 'experiment'. You can override using the `.groups` argument.
experiment default epsilon_greedy frrmab linucb pm random-2_4 random-2_4_6 random-4_6 random-4_8 ts random random-10 random-2 random-3 random-5 adapt-all-arpd-final adapt-no-local-search-final ts-perturb
01-adaptive_destruction_size 0.595 0.80 0.491 0.67 0.454 0.69 0.441 0.61 0.466 0.64 0.648 0.84 0.506 0.68 0.479 0.67 0.416 0.60 0.438 0.67 NA NA NA NA NA NA NA NA
02-adaptive_best_insertion 0.595 0.80 0.546 0.74 0.554 0.76 0.570 0.76 0.583 0.80 NA NA NA NA 0.556 0.76 NA NA NA NA NA NA NA NA
03-adaptive_destruction_position 0.594 0.80 0.575 0.75 0.553 0.73 0.672 0.79 0.563 0.73 NA NA NA NA 0.548 0.73 NA NA NA NA NA NA NA NA
04-adaptive_local_search 0.595 0.80 0.686 1.00 0.666 0.94 0.696 1.00 0.686 1.00 NA NA NA NA 0.662 0.96 0.694 1.00 NA NA NA NA NA NA NA
05-adaptive_perturb 0.595 0.80 0.377 0.62 0.353 0.61 0.434 0.70 0.351 0.63 NA NA NA NA 0.343 0.63 0.348 0.63 NA NA NA NA NA NA NA
06-adaptive_neighborhood_size 0.596 0.80 0.599 0.80 0.589 0.80 0.589 0.80 0.593 0.79 NA NA NA NA 0.598 0.81 NA 0.646 0.86 0.567 0.75 0.590 0.79 0.605 0.79 NA NA NA
09-final_comparison 0.595 0.80 NA NA NA NA NA NA NA NA NA NA NA NA NA NA 0.455 0.77 0.380 0.64 0.343 0.63
filter_ig_default <- function(config) {
  remove_param <- c(
    "IG.AdaptivePosition.AOS.WarmUp.Strategy",
    # "IG.AdaptivePosition.AOS.Options",
    "IG.AdaptivePosition.AOS.Strategy",
    "IG.AdaptiveBestInsertion.AOS.Strategy",
    "IG.AdaptivePerturb.AOS.Strategy",
    "IG.AdaptiveLocalSearch.AOS.Strategy",
    "IG.AdaptiveNeighborhoodSize.AOS.Strategy",
    "IG.AOS.Strategy"
  )
  config[(str_detect(names(config), "AOS") |
          str_detect(names(config), "AdaptiveBestInsertion") |
          str_detect(names(config), "AdaptivePosition")) &
        !(names(config) %in% remove_param)]
}

format_config <- function(config) {
  param_names <- names(config) %>%
    str_replace("PM\\.RewardType", "PMRewardType") %>%
    str_remove("^IG\\.(AdaptivePosition|AdaptiveLocalSearch|AdaptiveBestInsertion|AdaptiveNeighborhoodSize|AdaptivePerturb)\\.AOS\\.") %>%
    str_remove("^IG\\.AOS\\.") %>%
    str_remove("^IG\\.AdaptiveBestInsertion\\.") %>%
    str_remove("^IG\\.AdaptivePosition\\.") %>%
    str_remove("^(PM|FRRMAB|LINUCB|TS)\\.")
  paste(param_names, config, sep = " = ")
}

load_config <- function(config_path) {
  read_rds(config_path)[1,] %>%
    df_to_character() %>%
    filter_ig_default() %>%
    format_config()
}

load_configs <- function(irace_folder, path = "all") {
  tibble(
    path = path,
    full_path = file.path(irace_folder, path)
  ) %>%
    select(path, full_path) %>%
    mutate(
      configs = map(full_path, function(path) {
        tibble(algo = dir(path)) %>%
          filter(!str_detect(algo, "_log|_rec")) %>%
          mutate(config = map(file.path(path, algo), load_config))
      })
    ) %>%
    unnest(configs)
}

configs_tidy <- map_dfr(experiments, ~{
  here("reports", "aos", "data", .x) %>%
    file.path("irace") %>%
    load_configs() %>%
    mutate(exp = .x) %>%
    select(exp, algo, config)
}) %>%
  unnest(config) %>%
  separate(config, c("param", "value"), sep = " = ")
configs_tidy %>%
  filter((param %in% c("RewardType"))) %>%
  filter(!(param %in% c("WarmUp.Strategy"))) %>%  # pivot_wider(names_from = "param", values_from = "value") %>% 
  arrange(exp, algo, param) %>%
  mmtable(value) +
  header_top(algo) +
  header_left(exp)
igepsilon_greedy.rds igfrrmab.rds iglinucb.rds igpm.rds igts.rds
01-adaptive_destruction_size 0 1 2 1 2
03-adaptive_destruction_position 2 3 3 0 1
04-adaptive_local_search 2 2 2 0 1
05-adaptive_perturb 3 1 3 0 1
06-adaptive_neighborhood_size 1 0 0 3 2
configs_tidy %>%
  filter((param %in% c("Options"))) %>% 
  filter(!(param %in% c("WarmUp.Strategy"))) %>%  # pivot_wider(names_from = "param", values_from = "value") %>%
  arrange(exp, algo, param) %>%
  mmtable(value) +
  header_top(algo) +
  header_left(exp)
igepsilon_greedy.rds igfrrmab.rds iglinucb.rds igpm.rds igts.rds
01-adaptive_destruction_size 2_4_6 4_8 4_8 4_6 4_8
configs_tidy %>%
  filter((param %in% c("NoArms"))) %>% 
  filter(!(param %in% c("WarmUp.Strategy"))) %>%  # pivot_wider(names_from = "param", values_from = "value") %>%
  arrange(exp, algo, param) %>%
  mmtable(value) +
  header_top(algo) +
  header_left(exp)
igepsilon_greedy.rds igfrrmab.rds iglinucb.rds igpm.rds igts.rds
02-adaptive_best_insertion fixed_3 fixed_10 fixed_3 no_jobs fixed_10
03-adaptive_destruction_position fixed_3 fixed_3 fixed_3 fixed_3 fixed_10
06-adaptive_neighborhood_size 3 2 2 3 2
configs_tidy %>%
  filter((param %in% c("WindowSize", "UpdateWindow", "C"))) %>% 
  filter(!(param %in% c("WarmUp.Strategy"))) %>%  # pivot_wider(names_from = "param", values_from = "value") %>%
  arrange(exp, algo, param) %>%
  mmtable(value) +
  header_top(algo) +
  header_left(exp)
igfrrmab.rds igpm.rds igts.rds
01-adaptive_destruction_size 406 128 NA
02-adaptive_best_insertion 271 125 299
03-adaptive_destruction_position 350 357 NA
04-adaptive_local_search 188 283 414
05-adaptive_perturb 59 283 21
06-adaptive_neighborhood_size 32 340 358
extras[[1]] %>%
  ggplot(aes(x = algo, fill = algo, y = arfp)) +
  geom_violin() + 
  labs(x = NULL) +
  scale_y_continuous(trans = "pseudo_log") +
  ggtitle("Adaptive perturb-ds") + 
  theme_minimal()

extras[[1]] %>%
  bind_rows() %>%
  group_by(experiment, algo) %>%
  summarise(
    arfp = sprintf("%.3f (%.3f)", mean(arfp), sd(arfp))
  ) %>%
  pivot_wider(names_from = "algo", values_from = "arfp") %>%
  knitr::kable(booktabs = TRUE)
#> `summarise()` has grouped output by 'experiment'. You can override using the `.groups` argument.
experiment default epsilon_greedy frrmab linucb pm random-2_4 random-2_4_6 random-4_6 random-4_8 ts
01-adaptive_destruction_size 0.938 (0.883) 0.761 (0.739) 0.702 (0.760) 0.685 (0.661) 0.720 (0.708) 0.985 (0.949) 0.804 (0.748) 0.757 (0.739) 0.648 (0.683) 0.686 (0.767)
algo_perfs <- perfs[["09-final_comparison"]]

algo_perfs %>%
  ggplot(aes(x = algo, fill = algo, y = arfp)) +
  geom_violin() + 
  labs(x = NULL) +
  scale_y_continuous(trans = "pseudo_log") +
  ggtitle("Adaptive perturb-ds") + 
  theme_minimal()

frdAllPairsNemenyiTest(algo_perfs$arfp, algo_perfs$algo, paste(algo_perfs$id, "-", algo_perfs$seed))
#> 
#>  Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> data: y, groups and blocks
#>                             adapt-all-arpd-final adapt-no-local-search-final
#> adapt-no-local-search-final 0.00072              -                          
#> default                     1.2e-09              3.5e-14                    
#> ts-perturb                  1.7e-09              0.06388                    
#>                             default
#> adapt-no-local-search-final -      
#> default                     -      
#> ts-perturb                  < 2e-16
#> 
#> P value adjustment method: single-step
perfs[["09-final_comparison"]] %>%
  bind_rows() %>%
  group_by(experiment, algo) %>%
  summarise(
    arfp = sprintf("%.3f (%.2f)", mean(arfp), sd(arfp))
  ) %>%
  pivot_wider(names_from = "algo", values_from = "arfp") %>%
  knitr::kable(booktabs = TRUE)
#> `summarise()` has grouped output by 'experiment'. You can override using the `.groups` argument.
experiment adapt-all-arpd-final adapt-no-local-search-final default ts-perturb
09-final_comparison 0.455 (0.77) 0.380 (0.64) 0.595 (0.80) 0.343 (0.63)
frdAllPairsNemenyiTest(perfs[["09-final_comparison"]]$arfp, perfs[["09-final_comparison"]]$algo, paste(perfs[["09-final_comparison"]]$id, "-", perfs[["09-final_comparison"]]$seed))
#> 
#>  Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> data: y, groups and blocks
#>                             adapt-all-arpd-final adapt-no-local-search-final
#> adapt-no-local-search-final 0.00072              -                          
#> default                     1.2e-09              3.5e-14                    
#> ts-perturb                  1.7e-09              0.06388                    
#>                             default
#> adapt-no-local-search-final -      
#> default                     -      
#> ts-perturb                  < 2e-16
#> 
#> P value adjustment method: single-step
perfs[["09-final_comparison"]] %>%
  bind_rows() %>%
  group_by(experiment, algo, objective) %>%
  summarise(
    arfp = sprintf("%.3f (%.2f)", mean(arfp), sd(arfp))
  ) %>%
  pivot_wider(names_from = "algo", values_from = "arfp") %>%
  knitr::kable(booktabs = TRUE)
#> `summarise()` has grouped output by 'experiment', 'algo'. You can override using the `.groups` argument.
experiment objective adapt-all-arpd-final adapt-no-local-search-final default ts-perturb
09-final_comparison FLOWTIME 0.592 (0.95) 0.466 (0.80) 0.763 (0.95) 0.436 (0.78)
09-final_comparison MAKESPAN 0.319 (0.49) 0.294 (0.42) 0.426 (0.56) 0.251 (0.40)
test_by_group <- function(perfs, group_var) {
  perfs %>% 
    group_by(across(group_var)) %>%
    group_split(.keep = TRUE) %>%
    map(~frdAllPairsNemenyiTest(.x$arfp, paste(.x$algo, .x[[group_var]]), paste(.x$id, "-", .x$seed)))
}

test_by_group(perfs[["09-final_comparison"]], 'objective')
#> Note: Using an external vector in selections is ambiguous.
#> ℹ Use `all_of(group_var)` instead of `group_var` to silence this message.
#> ℹ See <https://tidyselect.r-lib.org/reference/faq-external-vector.html>.
#> This message is displayed once per session.
#> [[1]]
#> 
#>  Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> data: y, groups and blocks
#>                                      adapt-all-arpd-final FLOWTIME
#> adapt-no-local-search-final FLOWTIME 0.001                        
#> default FLOWTIME                     7.7e-08                      
#> ts-perturb FLOWTIME                  2.8e-06                      
#>                                      adapt-no-local-search-final FLOWTIME
#> adapt-no-local-search-final FLOWTIME -                                   
#> default FLOWTIME                     3.6e-14                             
#> ts-perturb FLOWTIME                  0.571                               
#>                                      default FLOWTIME
#> adapt-no-local-search-final FLOWTIME -               
#> default FLOWTIME                     -               
#> ts-perturb FLOWTIME                  4.0e-14
#> 
#> P value adjustment method: single-step
#> 
#> [[2]]
#> 
#>  Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> 
#> data: y, groups and blocks
#>                                      adapt-all-arpd-final MAKESPAN
#> adapt-no-local-search-final MAKESPAN 0.33694                      
#> default MAKESPAN                     0.00512                      
#> ts-perturb MAKESPAN                  0.00059                      
#>                                      adapt-no-local-search-final MAKESPAN
#> adapt-no-local-search-final MAKESPAN -                                   
#> default MAKESPAN                     3.6e-06                             
#> ts-perturb MAKESPAN                  0.11979                             
#>                                      default MAKESPAN
#> adapt-no-local-search-final MAKESPAN -               
#> default MAKESPAN                     -               
#> ts-perturb MAKESPAN                  3.6e-12
#> 
#> P value adjustment method: single-step
perfs[["09-final_comparison"]] %>%
  bind_rows() %>%
  group_by(experiment, algo, type) %>%
  summarise(
    arfp = sprintf("%.3f (%.2f)", mean(arfp), sd(arfp))
  ) %>%
  pivot_wider(names_from = "algo", values_from = "arfp") %>%
  knitr::kable(booktabs = TRUE)
#> `summarise()` has grouped output by 'experiment', 'algo'. You can override using the `.groups` argument.
experiment type adapt-all-arpd-final adapt-no-local-search-final default ts-perturb
09-final_comparison NOIDLE 0.888 (1.07) 0.759 (0.90) 0.793 (0.92) 0.708 (0.90)
09-final_comparison NOWAIT 0.144 (0.26) 0.136 (0.25) 0.592 (0.76) 0.106 (0.22)
09-final_comparison PERM 0.334 (0.50) 0.245 (0.39) 0.400 (0.63) 0.216 (0.35)
test_by_group(perfs[["09-final_comparison"]], 'type')
#> [[1]]
#> 
#>  Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> data: y, groups and blocks
#>                                    adapt-all-arpd-final NOIDLE
#> adapt-no-local-search-final NOIDLE 0.01733                    
#> default NOIDLE                     0.94203                    
#> ts-perturb NOIDLE                  5e-05                      
#>                                    adapt-no-local-search-final NOIDLE
#> adapt-no-local-search-final NOIDLE -                                 
#> default NOIDLE                     0.08233                           
#> ts-perturb NOIDLE                  0.42803                           
#>                                    default NOIDLE
#> adapt-no-local-search-final NOIDLE -             
#> default NOIDLE                     -             
#> ts-perturb NOIDLE                  0.00058
#> 
#> P value adjustment method: single-step
#> 
#> [[2]]
#> 
#>  Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> 
#> data: y, groups and blocks
#>                                    adapt-all-arpd-final NOWAIT
#> adapt-no-local-search-final NOWAIT 0.84                       
#> default NOWAIT                     2.9e-14                    
#> ts-perturb NOWAIT                  0.12                       
#>                                    adapt-no-local-search-final NOWAIT
#> adapt-no-local-search-final NOWAIT -                                 
#> default NOWAIT                     3.7e-14                           
#> ts-perturb NOWAIT                  0.51                              
#>                                    default NOWAIT
#> adapt-no-local-search-final NOWAIT -             
#> default NOWAIT                     -             
#> ts-perturb NOWAIT                  < 2e-16
#> 
#> P value adjustment method: single-step
#> 
#> [[3]]
#> 
#>  Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> 
#> data: y, groups and blocks
#>                                  adapt-all-arpd-final PERM
#> adapt-no-local-search-final PERM 0.02086                  
#> default PERM                     0.20959                  
#> ts-perturb PERM                  0.00011                  
#>                                  adapt-no-local-search-final PERM default PERM
#> adapt-no-local-search-final PERM -                                -           
#> default PERM                     8.4e-06                          -           
#> ts-perturb PERM                  0.50361                          3.1e-09
#> 
#> P value adjustment method: single-step
perfs[["09-final_comparison"]] %>%
  bind_rows() %>%
  group_by(experiment, algo, dist, corr) %>%
  summarise(
    arfp = sprintf("%.3f (%.2f)", mean(arfp), sd(arfp))
  ) %>%
  pivot_wider(names_from = "algo", values_from = "arfp") %>%
  knitr::kable(booktabs = TRUE)
#> `summarise()` has grouped output by 'experiment', 'algo', 'dist'. You can override using the `.groups` argument.
experiment dist corr adapt-all-arpd-final adapt-no-local-search-final default ts-perturb
09-final_comparison exponential random 0.805 (1.05) 0.614 (0.82) 1.093 (0.99) 0.614 (0.84)
09-final_comparison uniform job-correlated 0.317 (0.52) 0.286 (0.50) 0.403 (0.53) 0.269 (0.48)
09-final_comparison uniform machine-correlated 0.042 (0.08) 0.031 (0.06) 0.070 (0.13) 0.030 (0.07)
09-final_comparison uniform random 0.658 (0.78) 0.589 (0.72) 0.814 (0.81) 0.461 (0.67)
test_by_group(perfs[["09-final_comparison"]] %>% mutate(dist_corr = paste(dist, corr)), 'dist_corr')
#> [[1]]
#> 
#>  Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> data: y, groups and blocks
#>                                                adapt-all-arpd-final exponential random
#> adapt-no-local-search-final exponential random 0.0158                                 
#> default exponential random                     0.0107                                 
#> ts-perturb exponential random                  0.0052                                 
#>                                                adapt-no-local-search-final exponential random
#> adapt-no-local-search-final exponential random -                                             
#> default exponential random                     8.0e-09                                       
#> ts-perturb exponential random                  0.9870                                        
#>                                                default exponential random
#> adapt-no-local-search-final exponential random -                         
#> default exponential random                     -                         
#> ts-perturb exponential random                  9.4e-10
#> 
#> P value adjustment method: single-step
#> 
#> [[2]]
#> 
#>  Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> 
#> data: y, groups and blocks
#>                                                    adapt-all-arpd-final uniform job-correlated
#> adapt-no-local-search-final uniform job-correlated 0.8799                                     
#> default uniform job-correlated                     0.0003                                     
#> ts-perturb uniform job-correlated                  0.2024                                     
#>                                                    adapt-no-local-search-final uniform job-correlated
#> adapt-no-local-search-final uniform job-correlated -                                                 
#> default uniform job-correlated                     9.9e-06                                           
#> ts-perturb uniform job-correlated                  0.6142                                            
#>                                                    default uniform job-correlated
#> adapt-no-local-search-final uniform job-correlated -                             
#> default uniform job-correlated                     -                             
#> ts-perturb uniform job-correlated                  1.1e-08
#> 
#> P value adjustment method: single-step
#> 
#> [[3]]
#> 
#>  Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> 
#> data: y, groups and blocks
#>                                                        adapt-all-arpd-final uniform machine-correlated
#> adapt-no-local-search-final uniform machine-correlated 0.1355                                         
#> default uniform machine-correlated                     0.0052                                         
#> ts-perturb uniform machine-correlated                  0.0730                                         
#>                                                        adapt-no-local-search-final uniform machine-correlated
#> adapt-no-local-search-final uniform machine-correlated -                                                     
#> default uniform machine-correlated                     2.8e-07                                               
#> ts-perturb uniform machine-correlated                  0.9935                                                
#>                                                        default uniform machine-correlated
#> adapt-no-local-search-final uniform machine-correlated -                                 
#> default uniform machine-correlated                     -                                 
#> ts-perturb uniform machine-correlated                  6.1e-08
#> 
#> P value adjustment method: single-step
#> 
#> [[4]]
#> 
#>  Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> 
#> data: y, groups and blocks
#>                                            adapt-all-arpd-final uniform random
#> adapt-no-local-search-final uniform random 0.27178                            
#> default uniform random                     0.10258                            
#> ts-perturb uniform random                  4.9e-06                            
#>                                            adapt-no-local-search-final uniform random
#> adapt-no-local-search-final uniform random -                                         
#> default uniform random                     0.00026                                   
#> ts-perturb uniform random                  0.00950                                   
#>                                            default uniform random
#> adapt-no-local-search-final uniform random -                     
#> default uniform random                     -                     
#> ts-perturb uniform random                  3.3e-12
#> 
#> P value adjustment method: single-step
perfs[["09-final_comparison"]] %>%
  bind_rows() %>%
  group_by(experiment, algo, no_jobs<=50) %>%
  summarise(
    arfp = sprintf("%.3f (%.2f)", mean(arfp), sd(arfp))
  ) %>%
  pivot_wider(names_from = "algo", values_from = "arfp") %>%
knitr::kable(booktabs = TRUE)
#> `summarise()` has grouped output by 'experiment', 'algo'. You can override using the `.groups` argument.
experiment no_jobs <= 50 adapt-all-arpd-final adapt-no-local-search-final default ts-perturb
09-final_comparison FALSE 0.697 (0.89) 0.554 (0.76) 0.938 (0.88) 0.515 (0.75)
09-final_comparison TRUE 0.214 (0.52) 0.206 (0.45) 0.251 (0.50) 0.172 (0.39)
test_by_group(perfs[["09-final_comparison"]] %>% mutate(no_jobs = no_jobs <= 50), 'no_jobs')
#> [[1]]
#> 
#>  Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> data: y, groups and blocks
#>                                   adapt-all-arpd-final FALSE
#> adapt-no-local-search-final FALSE 3.4e-08                   
#> default FALSE                     1.1e-09                   
#> ts-perturb FALSE                  1.5e-13                   
#>                                   adapt-no-local-search-final FALSE
#> adapt-no-local-search-final FALSE -                                
#> default FALSE                     < 2e-16                          
#> ts-perturb FALSE                  0.26                             
#>                                   default FALSE
#> adapt-no-local-search-final FALSE -            
#> default FALSE                     -            
#> ts-perturb FALSE                  < 2e-16
#> 
#> P value adjustment method: single-step
#> 
#> [[2]]
#> 
#>  Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> 
#> data: y, groups and blocks
#>                                  adapt-all-arpd-final TRUE
#> adapt-no-local-search-final TRUE 0.97834                  
#> default TRUE                     0.04302                  
#> ts-perturb TRUE                  0.57891                  
#>                                  adapt-no-local-search-final TRUE default TRUE
#> adapt-no-local-search-final TRUE -                                -           
#> default TRUE                     0.11646                          -           
#> ts-perturb TRUE                  0.33694                          0.00056
#> 
#> P value adjustment method: single-step